'use strict'

var initDb = require('./utils/initDb')
// initDb();

var article = require('./models/articles');

var Crawler = require("crawler");

var fs = require("fs");

var request = require('request');

var c = new Crawler({
    rateLimit:200 ,
    
    maxConnections: 10,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            var $ = res.$;

            var list = $(".dw_table div[class ='el']").toArray();
            // console.log(list);
            list.forEach(element => {
                // var txt = $(element).find('.pl2 a ').text().trim().replace(/[ \r\n]/g, '');
                var txt1 =$(element).find('.t1 span').text().trim();
                var txt2 =$(element).find('.t2 a ').attr('title');
                var txt3 =$(element).find('.t3 ').text().trim();
                var txt4 =$(element).find('.t4 ').text().trim();
                var txt5 =$(element).find('.t5 ').text().trim();
                article.create({ title: txt1,companyName: txt2,siteName:txt3,salary: txt4,Time:txt5 });
                console.log(txt1,txt2,txt3,txt4,txt5);
              
            });

            // var $ = res.$;

            // var list = $('#content .article table').toArray();
            // console.log(list);
            // list.forEach(element => {
            //     var txt = $(element).find(' img').attr('src');
            //     var imgName = txt.split('/').slice(7,8).toString();
                
            //     console.log(txt);
            //     console.log(imgName);
            //     // require(txt).pipe(fs.createWriteStream('./images/'  + imgName + '.jpg'))
            //     request(txt).pipe(fs.createWriteStream('./images/'  + imgName + '.jpg'))
            //     article.create({title:txt});

            // });



        };
        done();
    }
});



for (var i = 0; i <= 15; i = i +1) {
    
    c.queue('https://search.51job.com/list/010000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590,2,'+i+'.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=');
}

